###########################################
#  Primary Divisions: How Voters Evaluate Policy and Group Differences in Intra-Party Contests
#   - Forthcoming at The Journal of Politics
#   - Henderson et al 2021
#
###########################################
#  - code by S. Goggin & J. Henderson
########################################################
# This file produces ideology models that are used to produce ideology scores assigned to candidates in the next file
#   (i.e., used in the ideology score data builds)
########################################################
# inputs :: /data/cces_stacked_unmatched.Rdata

# outputs ::
# => core_for_ggplot in data/core_for_ggplot_global.csv :: these results are saved so to replicate our figure ordering across all subsequent figures/results
# => core_for_ggplot in data/core_for_ggplot_dem_global.csv :: these results are saved so to replicate our figure ordering and scoring for dem candidates
# => core_for_ggplot in data/core_for_ggplot_rep_global.csv :: these results are saved so to replicate our figure ordering and scoring for rep candidates

# => data/scoreMats.Rdata object containing scores of candidates by party
# => new candidate_matrix in data/candidate_matrix_scored.csv with scoring variables included

#dirs="~/Dropbox/replication0/"
#dirs should be set here or in runR.R

rm(list=ls()[which(ls()!='dirs')])
library(ggplot2)
library(stringr)

# messy function to reorder by some estimate value
lableOrder=function(xmat,labels,label.groups,omits,o.column){

	# denote which label is to be omitted on the label
	for(i in 1:length(omits)){
		labels[which(labels==omits[i])]=paste('omit',labels[which(labels==omits[i])],sep='_')
	}

	# break groups into levels
	un_group=unique(label.groups)

	# this is the item to sort on, typically global or independent
	xm=xmat[,o.column]

	# vector which will contain row order
	xo=1:length(xm)

	# rearranging roworder within level
	for(j in 1:length(un_group)){
		ix=which(label.groups==un_group[j])
		if(length(ix)>2){
			ix=ix[!grepl(labels[ix],pattern='omit')]
			xo[ix]=xo[ix][order(xm[ix])]
		}
	}
	return(xmat[xo,])
}

reOrder=function(x,o){
	ix=array(NA,nrow(x))
	for(i in 1:length(o)){
		ix[i]=which(x$iv_order==o[i])
	}
	return(x[ix,])
}


setwd(dirs)

load("data/cces_stacked_unmatched.Rdata")
candidate_matrix=read.csv("data/candidate_matrix.csv",header=T,stringsAsFactors=F)[,-c(1)]


###########################################
###First, need to stack based on candidates, not just candidate pairs (and also get text out for labels later)

#This has leaners as independents, which is incorrect
#cces_stacked$pid3clean <- ifelse(cces_stacked$pid3=="Democrat",-1,ifelse(cces_stacked$pid3=="Republican",1,0))

library(car)

###########################################
###Then, produce models w/ Clustered SEs

#Function from: http://scholar.byu.edu/jgubler/book/clustered-standard-errors-r
#Need this for clustered standard errors
clse.f <- function(dat,fm, cluster){
 require(sandwich)
 require(lmtest)
 not <- attr(fm$model,"na.action")
if( ! is.null(not)){
  cluster <- cluster[-not]
    dat <- dat[-not,]
}
 with(dat,{
 M <- length(unique(cluster))
 N <- length(cluster)
 K <- fm$rank
 dfc <- (M/(M-1))*((N-1)/(N-K))
 uj <- apply(estfun(fm),2, function(x) tapply(x, cluster, sum));
 vcovCL <- dfc*sandwich(fm, meat=crossprod(uj)/N)
 coeftest(fm, vcovCL)
 }
 )
}


####

#omitting: male, white, no religion, attorney, decent, newspaper endorsements, record = helping constituents, raising taxes for both issues

ideo_extreme <- subset(candidate_matrix,candidate_matrix$conjoints==1|candidate_matrix$conjoints==2|candidate_matrix$conjoints==5|candidate_matrix$conjoints==6)

#Standardizing the direction across 1,2,5,6 JAHconjoints w/ Lib = 0, Cons = 1
ideo_extreme$dv_libcon <- ifelse(
ideo_extreme$conjoints==1& ideo_extreme$dv_choice==1,0,ifelse(
ideo_extreme$conjoints==1& ideo_extreme$dv_choice==0,1,ifelse(
ideo_extreme$conjoints==5& ideo_extreme$dv_choice==1,0,ifelse(
ideo_extreme$conjoints==5& ideo_extreme$dv_choice==0,1,ifelse(
ideo_extreme$conjoints==2& ideo_extreme$dv_choice==1,1,ifelse(
ideo_extreme$conjoints==2& ideo_extreme$dv_choice==0,0,ifelse(
ideo_extreme$conjoints==6& ideo_extreme$dv_choice==1,1,ifelse(
ideo_extreme$conjoints==6& ideo_extreme$dv_choice==0,0,NA))))))))

ideo_extreme$pty <- ifelse(ideo_extreme$conjoints==1 | ideo_extreme$conjoints==5,0,1)

#####Global Results

attach(ideo_extreme)
global_ideo <- lm(dv_libcon~
pty+g_female+
re_black+re_hispanic+
r_catholic+r_evangelical+r_protestant+
o_ceo+o_citycouncil+o_factoryforeman+o_farmer+o_usarmymajor+o_politicalstaffer+o_smallbizowner+o_stateleg+o_teacher+
p_compassionate+p_empathetic+p_inspiring+p_intelligent+p_knowledgeable+p_moral+p_strongleader+
e_business+e_christian+e_civilrights+e_energy+e_environment+e_guncontrol+e_gunrights+e_laborunions+e_reproductive+e_taxreform+e_teaparty+e_veterans+
rec_refuse+rec_secure+rec_stand+rec_work+
i_raisetaxes+i_cuttaxes+i_lgbt+i_marriage+i_drilling+i_need+i_govabuse+i_righttochoose+i_gunrights+i_unfairtrade+i_unbornlives+i_citizenship+i_reducemilitary+i_policing+i_co2emissions+i_bordersecurity+i_guncontrol+i_strengthenmilitary+i_criminals,
weights = 1/wt)

global_ideo_clse <- clse.f(ideo_extreme,global_ideo,respondent)
detach(ideo_extreme)

results_matrix <- cbind(global_ideo_clse[,1:2])
colnames(results_matrix) <- c("estimate","se")
results_matrix <- results_matrix[3:59,]
results_matrix_withnames <- cbind(var=rownames(results_matrix),results_matrix)
rownames(results_matrix_withnames) <- seq(1,57,by=1)

##Now inserting rows for the omitted levels

#omitted: male, white, no religion, attorney, decent, newspaper endorsements, record = helping constituents, raising taxes for both issues

g_male <- c("g_male",rep(0,2))
re_white <- c("re_white",rep(0,2))
r_none <- c("r_none",rep(0,2))
o_attorney <- c("o_attorney",rep(0,2))
p_decent <- c("p_decent",rep(0,2))
e_newspapers <- c("e_newspapers",rep(0,2))
rec_help <- c("rec_help",rep(0,2))
#i_raisetaxes <- c("i1_raisetaxes",rep(0,2))
i_freetrade <- c("i1_freetrade",rep(0,2))

full_matrix <- rbind(
g_male,
results_matrix_withnames[1,],
re_white,
results_matrix_withnames[2:3,],
r_none,
results_matrix_withnames[4:6,],
o_attorney,
results_matrix_withnames[7:15,],
p_decent,
results_matrix_withnames[16:22,],
e_newspapers,
results_matrix_withnames[23:34,],
rec_help,
results_matrix_withnames[35:38,],
i_freetrade,
results_matrix_withnames[39:57,]
)

rownames(full_matrix) <- full_matrix[,1]
full_matrix <- full_matrix[,2:ncol(full_matrix)]
full_matrix_clean <- apply(full_matrix,2,as.numeric)
rownames(full_matrix_clean) <- rownames(full_matrix)

labels <- c(
"Gender - Male",
"Gender - Female",
"Race - White",
"Race - Black",
"Race - Hispanic",
"Religion - None",
"Religion - Catholic",
"Religion - Evangelical Protestant",
"Religion - Protestant",
"Occupation - Attorney",
"Occupation - CEO",
"Occupation - City Council Member",
"Occupation - Factory Foreman",
"Occupation - Farmer",
"Occupation - Former US Army Major",
"Occupation - Political Staffer",
"Occupation - Small Business Owner",
"Occupation - State Legislator",
"Occupation - Teacher",
"Personality - Decent",
"Personality - Compassionate",
"Personality - Empathetic",
"Personality - Inspiring",
"Personality - Intelligent",
"Personality - Knowledgeable",
"Personality - Moral",
"Personality - Strong Leader",
"Endorsements - Major area newspapers",
"Endorsements - Business groups",
"Endorsements - Christian groups",
"Endorsements - Civil rights groups",
"Endorsements - Energy groups",
"Endorsements - Environmental groups",
"Endorsements - Gun control groups",
"Endorsements - Gun rights groups",
"Endorsements - Labor unions",
"Endorsements - Reproductive rights groups",
"Endorsements - Tax reform groups",
"Endorsements - Tea Party groups",
"Endorsements - Veterans groups",
"Record - Help my constituents get the benefits they deserve",
"Record - Refuse to compromise my principles even when it means taking on my party",
"Record - Secure appointment to a powerful legislative committee",
"Record - Stand with my party to do what's right",
"Record - Work across the aisle to get things done",
"Issue - Promote expanding free trade agreements",
"Issue - Raise taxes on those making more than $250,000 a year",
"Issue - Cut taxes on income and capital gains for all",
"Issue - Defend the rights of LGBT individuals",
"Issue - Defend traditional marriage and religious beliefs",
"Issue - Expand domestic oil and gas production through drilling",
"Issue - Expand government and unemployment assistance for those in need",
"Issue - Prevent and prosecute abuse of government assistance programs",
"Issue - Protect a woman's right to choose",
"Issue - Protect gun owners' rights to defend themselves and others",
"Issue - Protect jobs and industry from unfair foreign trade",
"Issue - Protect the lives of the unborn",
"Issue - Provide a path to citizenship for undocumented immigrants",
"Issue - Reduce the size of military and number of military bases",
"Issue - Reform policing and stop racial profiling",
"Issue - Regulate CO2 emissions to combat global warming",
"Issue - Strengthen border security to stop illegal immigration",
"Issue - Strengthen gun control through commonsense restrictions",
"Issue - Strengthen our military and national defense",
"Issue - Toughen sentences and penalties for criminals")

core_for_ggplot <- data.frame(labels=labels,full_matrix_clean)
core_for_ggplot$iv_order <- factor(core_for_ggplot$labels, as.character(core_for_ggplot$labels))

omits=c("Gender - Male",
"Race - White",
"Religion - None",
"Occupation - Attorney",
"Personality - Decent",
"Endorsements - Major area newspapers",
"Record - Help my constituents get the benefits they deserve",
"Issue - Promote expanding free trade agreements")

label.groups=str_sub(labels,1,str_locate(labels,pattern='-')[,1]-2)
o.column=grep(names(core_for_ggplot),pattern='est')

core_for_ggplot=lableOrder(xmat=core_for_ggplot,labels,label.groups,omits,o.column)

write.csv(core_for_ggplot,"data/core_for_ggplot_global.csv")

#END libcon_global.R
#BEGIN libcon_demcand_global.R

###########################################

rm(list=ls()[which(ls()!='dirs')])
library(ggplot2)
library(stringr)

# messy function to reorder by some estimate value
lableOrder=function(xmat,labels,label.groups,omits,o.column){

	# denote which label is to be omitted on the label
	for(i in 1:length(omits)){
		labels[which(labels==omits[i])]=paste('omit',labels[which(labels==omits[i])],sep='_')
	}

	# break groups into levels
	un_group=unique(label.groups)

	# this is the item to sort on, typically global or independent
	xm=xmat[,o.column]

	# vector which will contain row order
	xo=1:length(xm)

	# rearranging roworder within level
	for(j in 1:length(un_group)){
		ix=which(label.groups==un_group[j])
		if(length(ix)>2){
			ix=ix[!grepl(labels[ix],pattern='omit')]
			xo[ix]=xo[ix][order(xm[ix])]
		}
	}
	return(xmat[xo,])
}

reOrder=function(x,o){
	ix=array(NA,nrow(x))
	for(i in 1:length(o)){
		ix[i]=which(x$iv_order==o[i])
	}
	return(x[ix,])
}

load("data/cces_stacked_unmatched.Rdata")
candidate_matrix=read.csv("data/candidate_matrix.csv",header=T,stringsAsFactors=F)[,-c(1)]

###########################################
###First, need to stack based on candidates, not just candidate pairs (and also get text out for labels later)

#This has leaners as independents, which is incorrect
#cces_stacked$pid3clean <- ifelse(cces_stacked$pid3=="Democrat",-1,ifelse(cces_stacked$pid3=="Republican",1,0))

library(car)

###########################################
###Then, produce models w/ Clustered SEs

#Function from: http://scholar.byu.edu/jgubler/book/clustered-standard-errors-r
#Need this for clustered standard errors
clse.f <- function(dat,fm, cluster){
 require(sandwich)
 require(lmtest)
 not <- attr(fm$model,"na.action")
if( ! is.null(not)){
  cluster <- cluster[-not]
    dat <- dat[-not,]
}
 with(dat,{
 M <- length(unique(cluster))
 N <- length(cluster)
 K <- fm$rank
 dfc <- (M/(M-1))*((N-1)/(N-K))
 uj <- apply(estfun(fm),2, function(x) tapply(x, cluster, sum));
 vcovCL <- dfc*sandwich(fm, meat=crossprod(uj)/N)
 coeftest(fm, vcovCL)
 }
 )
}


####

#############################################################
#########NOW THE IDEOLOGY RESULTS BY Candidate Party
#conjoints 1 & 5 = Dem, 2 & 6 = Rep

ideo_extreme <- subset(candidate_matrix,candidate_matrix$conjoints==1|candidate_matrix$conjoints==5)

#Standardizing the direction across 1,2,5,6 JAHconjoints w/ Lib = 0, Cons = 1
ideo_extreme$dv_libcon <- ifelse(
ideo_extreme$conjoints==1& ideo_extreme$dv_choice==1,0,ifelse(
ideo_extreme$conjoints==1& ideo_extreme$dv_choice==0,1,ifelse(
ideo_extreme$conjoints==5& ideo_extreme$dv_choice==1,0,ifelse(
ideo_extreme$conjoints==5& ideo_extreme$dv_choice==0,1,ifelse(
ideo_extreme$conjoints==2& ideo_extreme$dv_choice==1,1,ifelse(
ideo_extreme$conjoints==2& ideo_extreme$dv_choice==0,0,ifelse(
ideo_extreme$conjoints==6& ideo_extreme$dv_choice==1,1,ifelse(
ideo_extreme$conjoints==6& ideo_extreme$dv_choice==0,0,NA))))))))

attach(ideo_extreme)
global_ideo <- lm(dv_libcon~
g_female+
re_black+re_hispanic+
r_catholic+r_evangelical+r_protestant+
o_ceo+o_citycouncil+o_factoryforeman+o_farmer+o_usarmymajor+o_politicalstaffer+o_smallbizowner+o_stateleg+o_teacher+
p_compassionate+p_empathetic+p_inspiring+p_intelligent+p_knowledgeable+p_moral+p_strongleader+
e_business+e_christian+e_civilrights+e_energy+e_environment+e_guncontrol+e_gunrights+e_laborunions+e_reproductive+e_taxreform+e_teaparty+e_veterans+
rec_refuse+rec_secure+rec_stand+rec_work+
i_raisetaxes+i_cuttaxes+i_lgbt+i_marriage+i_drilling+i_need+i_govabuse+i_righttochoose+i_gunrights+i_unfairtrade+i_unbornlives+i_citizenship+i_reducemilitary+i_policing+i_co2emissions+i_bordersecurity+i_guncontrol+i_strengthenmilitary+i_criminals,
weights = 1/wt)

global_ideo_clse <- clse.f(ideo_extreme,global_ideo,respondent)
detach(ideo_extreme)

results_matrix <- cbind(global_ideo_clse[,1:2])
colnames(results_matrix) <- c("estimate","se")
results_matrix <- results_matrix[2:50,]
results_matrix_withnames <- cbind(var=rownames(results_matrix),results_matrix)
rownames(results_matrix_withnames) <- seq(1,49,by=1)


##Now inserting rows for the omitted levels

#omitted: male, white, no religion, attorney, decent, newspaper endorsements, record = helping constituents, raising taxes for both issues

g_male <- c("g_male",rep(0,2))
re_white <- c("re_white",rep(0,2))
r_none <- c("r_none",rep(0,2))
o_attorney <- c("o_attorney",rep(0,2))
p_decent <- c("p_decent",rep(0,2))
e_newspapers <- c("e_newspapers",rep(0,2))
rec_help <- c("rec_help",rep(0,2))
i_freetrade <- c("i1_freetrade",rep(0,2))

full_matrix <- rbind(
g_male,
results_matrix_withnames[1,],
re_white,
results_matrix_withnames[2:3,],
r_none,
results_matrix_withnames[4:6,],
o_attorney,
results_matrix_withnames[7:15,],
p_decent,
results_matrix_withnames[16:22,],
e_newspapers,
results_matrix_withnames[23:29,],
rec_help,
results_matrix_withnames[30:33,],
i_freetrade,
results_matrix_withnames[34:49,]
)

full_matrix=rbind(full_matrix,matrix(NA,8,dim(full_matrix)[2]))

full_matrix[which(is.na(full_matrix[,1]))]=c(
	'e_taxreform',
	'e_teaparty',
	'e_christian',
	'e_energy',
	'e_gunrights',
	'i_unbornlives',
	'i_marriage',
	'i_gunrights')

rownames(full_matrix) <- full_matrix[,1]
full_matrix <- full_matrix[,2:ncol(full_matrix)]
full_matrix_clean <- apply(full_matrix,2,as.numeric)
rownames(full_matrix_clean) <- rownames(full_matrix)

dem_labels <- c(
"Gender - Male",
"Gender - Female",
"Race - White",
"Race - Black",
"Race - Hispanic",
"Religion - None",
"Religion - Catholic",
"Religion - Evangelical Protestant",
"Religion - Protestant",
"Occupation - Attorney",
"Occupation - CEO",
"Occupation - City Council Member",
"Occupation - Factory Foreman",
"Occupation - Farmer",
"Occupation - Former US Army Major",
"Occupation - Political Staffer",
"Occupation - Small Business Owner",
"Occupation - State Legislator",
"Occupation - Teacher",
"Personality - Decent",
"Personality - Compassionate",
"Personality - Empathetic",
"Personality - Inspiring",
"Personality - Intelligent",
"Personality - Knowledgeable",
"Personality - Moral",
"Personality - Strong Leader",
"Endorsements - Major area newspapers",
"Endorsements - Business groups",
"Endorsements - Civil rights groups",
"Endorsements - Environmental groups",
"Endorsements - Gun control groups",
"Endorsements - Labor unions",
"Endorsements - Reproductive rights groups",
"Endorsements - Veterans groups",
"Record - Help my constituents get the benefits they deserve",
"Record - Refuse to compromise my principles even when it means taking on my party",
"Record - Secure appointment to a powerful legislative committee",
"Record - Stand with my party to do what's right",
"Record - Work across the aisle to get things done",
"Issue - Promote expanding free trade agreements",
"Issue - Raise taxes on those making more than $250,000 a year",
"Issue - Cut taxes on income and capital gains for all",
"Issue - Defend the rights of LGBT individuals",
"Issue - Expand domestic oil and gas production through drilling",
"Issue - Expand government and unemployment assistance for those in need",
"Issue - Prevent and prosecute abuse of government assistance programs",
"Issue - Protect a woman's right to choose",
"Issue - Protect jobs and industry from unfair foreign trade",
"Issue - Provide a path to citizenship for undocumented immigrants",
"Issue - Reduce the size of military and number of military bases",
"Issue - Reform policing and stop racial profiling",
"Issue - Regulate CO2 emissions to combat global warming",
"Issue - Strengthen border security to stop illegal immigration",
"Issue - Strengthen gun control through commonsense restrictions",
"Issue - Strengthen our military and national defense",
"Issue - Toughen sentences and penalties for criminals",
"Endorsements - Tax reform groups",
"Endorsements - Tea Party groups",
"Endorsements - Christian groups",
"Endorsements - Energy groups",
"Endorsements - Gun rights groups",
"Issue - Protect the lives of the unborn",
"Issue - Defend traditional marriage and religious beliefs",
"Issue - Protect gun owners' rights to defend themselves and others")

core_for_ggplot <- data.frame(labels=dem_labels,full_matrix_clean)
core_for_ggplot$iv_order <- factor(core_for_ggplot$labels, as.character(core_for_ggplot$labels))

omits=c("Gender - Male",
"Race - White",
"Religion - None",
"Occupation - Attorney",
"Personality - Decent",
"Endorsements - Major area newspapers",
"Record - Help my constituents get the benefits they deserve",
"Issue - Promote expanding free trade agreements")

#labels=dem_labels
label.groups=str_sub(dem_labels,1,str_locate(dem_labels,pattern='-')[,1]-2)
o.column=grep(names(core_for_ggplot),pattern='est')

#core_for_ggplot=lableOrder(xmat=core_for_ggplot,labels=dem_labels,label.groups,omits,o.column)

write.csv(core_for_ggplot,"data/core_for_ggplot_dem_global.csv")

#END libcon_demcand.R
#BEGIN libcon_repcand.R

###########################################

rm(list=ls()[which(ls()!='dirs')])
library(ggplot2)
library(stringr)

# messy function to reorder by some estimate value
lableOrder=function(xmat,labels,label.groups,omits,o.column){

	# denote which label is to be omitted on the label
	for(i in 1:length(omits)){
		labels[which(labels==omits[i])]=paste('omit',labels[which(labels==omits[i])],sep='_')
	}

	# break groups into levels
	un_group=unique(label.groups)

	# this is the item to sort on, typically global or independent
	xm=xmat[,o.column]

	# vector which will contain row order
	xo=1:length(xm)

	# rearranging roworder within level
	for(j in 1:length(un_group)){
		ix=which(label.groups==un_group[j])
		if(length(ix)>2){
			ix=ix[!grepl(labels[ix],pattern='omit')]
			xo[ix]=xo[ix][order(xm[ix])]
		}
	}
	return(xmat[xo,])
}

reOrder=function(x,o){
	ix=array(NA,nrow(x))
	for(i in 1:length(o)){
		ix[i]=which(x$iv_order==o[i])
	}
	return(x[ix,])
}

load("data/cces_stacked_unmatched.Rdata")
candidate_matrix=read.csv("data/candidate_matrix.csv",header=T,stringsAsFactors=F)[,-c(1)]

###########################################
###First, need to stack based on candidates, not just candidate pairs (and also get text out for labels later)

#This has leaners as independents, which is incorrect
#cces_stacked$pid3clean <- ifelse(cces_stacked$pid3=="Democrat",-1,ifelse(cces_stacked$pid3=="Republican",1,0))

library(car)

###########################################
###Then, produce models w/ Clustered SEs

#Function from: http://scholar.byu.edu/jgubler/book/clustered-standard-errors-r
#Need this for clustered standard errors
clse.f <- function(dat,fm, cluster){
 require(sandwich)
 require(lmtest)
 not <- attr(fm$model,"na.action")
if( ! is.null(not)){
  cluster <- cluster[-not]
    dat <- dat[-not,]
}
 with(dat,{
 M <- length(unique(cluster))
 N <- length(cluster)
 K <- fm$rank
 dfc <- (M/(M-1))*((N-1)/(N-K))
 uj <- apply(estfun(fm),2, function(x) tapply(x, cluster, sum));
 vcovCL <- dfc*sandwich(fm, meat=crossprod(uj)/N)
 coeftest(fm, vcovCL)
 }
 )
}


####

#############################################################
#########NOW THE IDEOLOGY RESULTS BY Candidate Party
ideo_extreme <- subset(candidate_matrix,candidate_matrix$conjoints==2|candidate_matrix$conjoints==6)

#Standardizing the direction across 1,2,5,6 JAHconjoints w/ Lib = 0, Cons = 1
ideo_extreme$dv_libcon <- ifelse(
ideo_extreme$conjoints==1& ideo_extreme$dv_choice==1,0,ifelse(
ideo_extreme$conjoints==1& ideo_extreme$dv_choice==0,1,ifelse(
ideo_extreme$conjoints==5& ideo_extreme$dv_choice==1,0,ifelse(
ideo_extreme$conjoints==5& ideo_extreme$dv_choice==0,1,ifelse(
ideo_extreme$conjoints==2& ideo_extreme$dv_choice==1,1,ifelse(
ideo_extreme$conjoints==2& ideo_extreme$dv_choice==0,0,ifelse(
ideo_extreme$conjoints==6& ideo_extreme$dv_choice==1,1,ifelse(
ideo_extreme$conjoints==6& ideo_extreme$dv_choice==0,0,NA))))))))


attach(ideo_extreme)
global_ideo <- lm(dv_libcon~
g_female+
re_black+re_hispanic+
r_catholic+r_evangelical+r_protestant+
o_ceo+o_citycouncil+o_factoryforeman+o_farmer+o_usarmymajor+o_politicalstaffer+o_smallbizowner+o_stateleg+o_teacher+
p_compassionate+p_empathetic+p_inspiring+p_intelligent+p_knowledgeable+p_moral+p_strongleader+
e_business+e_christian+e_civilrights+e_energy+e_environment+e_guncontrol+e_gunrights+e_laborunions+e_reproductive+e_taxreform+e_teaparty+e_veterans+
rec_refuse+rec_secure+rec_stand+rec_work+
i_raisetaxes+i_cuttaxes+i_lgbt+i_marriage+i_drilling+i_need+i_govabuse+i_righttochoose+i_gunrights+i_unfairtrade+i_unbornlives+i_citizenship+i_reducemilitary+i_policing+i_co2emissions+i_bordersecurity+i_guncontrol+i_strengthenmilitary+i_criminals,
weights = 1/wt)

global_ideo_clse <- clse.f(ideo_extreme,global_ideo,respondent)
detach(ideo_extreme)

results_matrix <- cbind(global_ideo_clse[,1:2])
colnames(results_matrix) <- c("estimate","se")
results_matrix <- results_matrix[2:50,]
results_matrix_withnames <- cbind(var=rownames(results_matrix),results_matrix)
rownames(results_matrix_withnames) <- seq(1,49,by=1)


##Now inserting rows for the omitted levels

#omitted: male, white, no religion, attorney, decent, newspaper endorsements, record = helping constituents, raising taxes for both issues

g_male <- c("g_male",rep(0,2))
re_white <- c("re_white",rep(0,2))
r_none <- c("r_none",rep(0,2))
o_attorney <- c("o_attorney",rep(0,2))
p_decent <- c("p_decent",rep(0,2))
e_newspapers <- c("e_newspapers",rep(0,2))
rec_help <- c("rec_help",rep(0,2))
i_freetrade <- c("i1_freetrade",rep(0,2))

full_matrix <- rbind(
g_male,
results_matrix_withnames[1,],
re_white,
results_matrix_withnames[2:3,],
r_none,
results_matrix_withnames[4:6,],
o_attorney,
results_matrix_withnames[7:15,],
p_decent,
results_matrix_withnames[16:22,],
e_newspapers,
results_matrix_withnames[23:29,],
rec_help,
results_matrix_withnames[30:33,],
i_freetrade,
results_matrix_withnames[34:49,]
)

full_matrix=rbind(full_matrix,matrix(NA,8,dim(full_matrix)[2]))

full_matrix[which(is.na(full_matrix[,1]))]=c(
	'e_laborunions',
	'e_civilrights',
	'e_reproductive',
	'e_environment',
	'e_guncontrol',
	'i_righttochoose',
	'i_lgbt',
	'i_guncontrol')


rownames(full_matrix) <- full_matrix[,1]
full_matrix <- full_matrix[,2:ncol(full_matrix)]
full_matrix_clean <- apply(full_matrix,2,as.numeric)
rownames(full_matrix_clean) <- rownames(full_matrix)

rep_labels <- c(
"Gender - Male",
"Gender - Female",
"Race - White",
"Race - Black",
"Race - Hispanic",
"Religion - None",
"Religion - Catholic",
"Religion - Evangelical Protestant",
"Religion - Protestant",
"Occupation - Attorney",
"Occupation - CEO",
"Occupation - City Council Member",
"Occupation - Factory Foreman",
"Occupation - Farmer",
"Occupation - Former US Army Major",
"Occupation - Political Staffer",
"Occupation - Small Business Owner",
"Occupation - State Legislator",
"Occupation - Teacher",
"Personality - Decent",
"Personality - Compassionate",
"Personality - Empathetic",
"Personality - Inspiring",
"Personality - Intelligent",
"Personality - Knowledgeable",
"Personality - Moral",
"Personality - Strong Leader",
"Endorsements - Major area newspapers",
"Endorsements - Business groups",
"Endorsements - Christian groups",
"Endorsements - Energy groups",
"Endorsements - Gun rights groups",
"Endorsements - Tax reform groups",
"Endorsements - Tea Party groups",
"Endorsements - Veterans groups",
"Record - Help my constituents get the benefits they deserve",
"Record - Refuse to compromise my principles even when it means taking on my party",
"Record - Secure appointment to a powerful legislative committee",
"Record - Stand with my party to do what's right",
"Record - Work across the aisle to get things done",
"Issue - Promote expanding free trade agreements",
"Issue - Raise taxes on those making more than $250,000 a year",
"Issue - Cut taxes on income and capital gains for all",
"Issue - Defend traditional marriage and religious beliefs",
"Issue - Expand domestic oil and gas production through drilling",
"Issue - Expand government and unemployment assistance for those in need",
"Issue - Prevent and prosecute abuse of government assistance programs",
"Issue - Protect gun owners' rights to defend themselves and others",
"Issue - Protect jobs and industry from unfair foreign trade",
"Issue - Protect the lives of the unborn",
"Issue - Provide a path to citizenship for undocumented immigrants",
"Issue - Reduce the size of military and number of military bases",
"Issue - Reform policing and stop racial profiling",
"Issue - Regulate CO2 emissions to combat global warming",
"Issue - Strengthen border security to stop illegal immigration",
"Issue - Strengthen our military and national defense",
"Issue - Toughen sentences and penalties for criminals",
"Endorsements - Labor unions",
"Endorsements - Civil rights groups",
"Endorsements - Reproductive rights groups",
"Endorsements - Environmental groups",
"Endorsements - Gun control groups",
"Issue - Protect a woman's right to choose",
"Issue - Defend the rights of LGBT individuals",
"Issue - Strengthen gun control through commonsense restrictions")

core_for_ggplot <- data.frame(labels=rep_labels,full_matrix_clean)
core_for_ggplot$iv_order <- factor(core_for_ggplot$labels, as.character(core_for_ggplot$labels))

omits=c("Gender - Male",
"Race - White",
"Religion - None",
"Occupation - Attorney",
"Personality - Decent",
"Endorsements - Major area newspapers",
"Record - Help my constituents get the benefits they deserve",
"Issue - Promote expanding free trade agreements")

#labels=rep_labels
label.groups=str_sub(rep_labels,1,str_locate(rep_labels,pattern='-')[,1]-2)
o.column=grep(names(core_for_ggplot),pattern='est')

#core_for_ggplot=lableOrder(xmat=core_for_ggplot,labels=rep_labels,label.groups,omits,o.column)

write.csv(core_for_ggplot,"data/core_for_ggplot_rep_global.csv")

# END 1_build_score_libcon.R

# trying to keep everything together in the data
# -- cces_stacked in data/cces_stacked_unmatched.Rdata
# -- voter_matrix in data/voter_matrix.csv
# -- candidate_matrix in data/candidate_matrix.csv
# -- core_for_ggplot in data/core_for_ggplot_rep_global.csv
# -- core_for_ggplot in data/core_for_ggplot_dem_global.csv (for dems)
# -- core_for_ggplot in data/core_for_ggplot_rep_global.csv (for reps)
